package com.simoniu.scalademo

import org.apache.spark.{SparkConf, SparkContext}

/**
 * 需求：设置并行度
 * 1：可以在textFile或者parallelize等方法的第二个参数中设置并行度
 * 2：或者通过spark.default.parallelism参数统一设置并行度
 * Created by simoniu
 */

object MoreParallelismScalaDemo {
  def main(args: Array[String]): Unit = {
    val conf = new SparkConf()
    conf.setAppName("MoreParallelismScala")

    //设置全局并行度
    //conf.set("spark.default.parallelism", "5")

    val sc = new SparkContext(conf)

    val dataRDD = sc.parallelize(Array("hello", "linux", "hello", "java", "hehe", "hello", "hadoop", "hello", "spark", "hehe"))
    dataRDD.map((_, 1))
      .reduceByKey(_ + _)
      .foreach(println(_))
    sc.stop()
  }
}
